package shining365.programmableweb.data.collect;

import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;

import org.apache.commons.configuration.Configuration;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.log4j.Logger;
import org.cyberneko.html.parsers.SAXParser;
import org.dom4j.Document;
import org.dom4j.DocumentException;
import org.dom4j.io.SAXReader;
import org.xml.sax.XMLReader;

import shining365.programmableweb.data.common.Utils;

/**
 * 集中管理页面清洗与解析
 * */
public class Foxtidy {
	
	public Foxtidy() {
		this(false, false);
	}
	
	public Foxtidy(boolean ns, boolean script) {
		this.ns = ns;
		this.script = script;
	}
	
	public String getPage(String url) throws IOException {
		String page = getRawPage(url);
		if (!ns) {
			page = clearXmlns(page);
		}
		if (!script) {
			page = clearScript(page);
		}
		return page;
	}
	
	public Document getPageDocument(String url) throws IOException, DocumentException {
		return htm2Dom(getPage(url));
	}

	/**返回不经修改的原始页面*/
	private static String getRawPage(String url) throws IOException {
		HttpClient httpClient = new HttpClient();
		httpClient.getHostConfiguration().setProxy(
				config.getString("foxtidy.host"), 
				Integer.parseInt(config.getString("foxtidy.port")));
		
		GetMethod urlGet = new GetMethod(url);
		int retCode;
		if ((retCode = httpClient.executeMethod(urlGet)) != HttpStatus.SC_OK) {
			throw new RuntimeException("Unexpected HTTP code: " + retCode);
		}
		
		try {
			return urlGet.getResponseBodyAsString();
		} finally {
			urlGet.releaseConnection();
		}
	}
	
	private static String clearXmlns(String page) {
		return page.replaceAll("xmlns=(\"|').*?\\1", "");
	}

	private static String clearScript(String page) {
		return page.replaceAll("(?s)<(script|SCRIPT)[^>]*>.*?</(script|SCRIPT)>", "");
	}

	private static Document htm2Dom(String htm) throws DocumentException, IOException {
		XMLReader xmlReader = new SAXParser();
		SAXReader reader = new SAXReader(xmlReader);
		Reader r = new StringReader(htm);
		try {
			return reader.read(r);
		} finally {
			r.close();
		}
	}

	private final boolean ns;
	private final boolean script;
	private static final Configuration config = Utils.getConfiguration();
	@SuppressWarnings("unused")
	private static final Logger	logger = Logger.getLogger(Foxtidy.class);
	
}
